from celery import task
from adlist.models import ToutiaoModel, RadarModel, SinaModel, SouhuModel, FenghuangModel, FeedsradarModel, \
    FeedsradarParameter, ToutiaoPatameter, SinaPatameter, RadarPatameter, SouhuPatameter, FenghuangPatameter
from ad_manage.models import Advertisement, RandomData
from django.http import HttpResponse, JsonResponse
import json

import logging
import random
from django.db.models.functions.base import Func
import urllib
from IMAGE_QUERY.extract_cnn_vgg16_keras import VGGNet
import datetime
from django.db.models import Q
import ssl
from django.views.decorators.csrf import csrf_exempt
from extra_apps.IMAGE_QUERY import extract
import pickle
from adlist.controller import toutiao_object, fenghuang_object, sina_object, souhu_object, feedsradar_object, \
    radar_object


#

@csrf_exempt
@task
def random_tasks():
    randomData = RandomData()
    industryStr = ""
    zoneStr = ""
    ageStr = ""
    for i in range(7):
        ind = random.randint(0, 10000)
        z = random.randint(0, 10000)
        a = random.randint(0, 10000)
        if i != 6:
            industryStr = industryStr + str(ind) + ","
            ageStr = ageStr + str(a) + ","
        zoneStr = zoneStr + str(z) + ","

    industryStr = industryStr[0:len(industryStr) - 1]
    zoneStr = zoneStr[0:len(zoneStr) - 1]
    ageStr = ageStr[0:len(ageStr) - 1]
    randomData.industry_random_data = industryStr
    randomData.zone_random_data = zoneStr
    randomData.age_random_data = ageStr
    randomData.save()


class CustomeFunc(Func):
    function = 'DATE_FORMAT'
    template = '%(function)s(%(expressions)s,%(format)s)'

    def __init__(self, expression, **extra):
        expressions = [expression]
        super(CustomeFunc, self).__init__(*expressions, **extra)



@csrf_exempt
@task
def fenghuang_new_task():
    create_ad = Advertisement.objects.values('create_time').filter(model_from='fenghuang_app').order_by('-create_time')[
                0:1]
    create_time = create_ad[0].get('create_time')
    fenghuangs = FenghuangModel.objects.filter(create_time__gt=create_time)
    if fenghuangs.count() > 0:
        fenghuang_create(fenghuangs)
    update_ad = Advertisement.objects.values('update_time').filter(model_from='fenghuang_app').order_by('-update_time')[
                0:1]
    update_time = update_ad[0].get('update_time')
    fenghuangs = FenghuangModel.objects.filter(update_time__gt=update_time)
    if fenghuangs.count() > 0:
        fenghuang_update(fenghuangs)


@csrf_exempt
@task
def sina_new_task(request):
    logger = logging.getLogger('django')
    logger.info("sina_task is starting")
    create_ad = Advertisement.objects.values('create_time').filter(model_from='sina_app').order_by('-create_time')[
                0:1]
    create_time = create_ad[0].get('create_time')
    sinas = SinaModel.objects.filter(create_time__gt=create_time)
    if sinas.count() > 0:
        sina_create(sinas)

    update_ad = Advertisement.objects.values('update_time').filter(model_from='sina_app').order_by('-update_time')[
                0:1]
    update_time = update_ad[0].get('update_time')
    sinas = SinaModel.objects.filter(update_time__gt=update_time)
    if sinas.count() > 0:
        sina_update(sinas)

    return JsonResponse({'json_data': 'success'})


@csrf_exempt
@task
def souhu_new_task():
    logger = logging.getLogger('django')
    logger.info("souhu_task is starting")
    # 新建 #
    create_ad = Advertisement.objects.values('create_time').filter(model_from='souhu_app').order_by('-create_time')[
                0:1]
    create_time = create_ad[0].get('create_time')
    souhus = SouhuModel.objects.filter(create_time__gt=create_time)
    if souhus.count() > 0:
        souhu_create(souhus)

    # 更新 #
    update_ad = Advertisement.objects.values('update_time').filter(model_from='souhu_app').order_by('-update_time')[
                0:1]
    update_time = update_ad[0].get('update_time')
    souhus = SouhuModel.objects.filter(update_time__gt=update_time)
    if souhus.count() > 0:
        souhu_update(souhus)


@csrf_exempt
@task
def toutiao_new_task():
    logger = logging.getLogger('django')
    logger.info("toutiao_task is starting")
    # 新增广告
    create_ad = Advertisement.objects.values('create_time').filter(model_from='toutiao_app').order_by('-create_time')[
                0:1]
    create_time = create_ad[0].get('create_time')
    toutiaos = ToutiaoModel.objects.filter(create_time__gt=create_time)
    if toutiaos.count() > 0:
        toutiao_create(toutiaos)

    update_ad = Advertisement.objects.values('update_time').filter(model_from='toutiao_app').order_by(
        '-update_time')[
                0:1]
    update_time = update_ad[0].get('update_time')
    toutiaos = ToutiaoModel.objects.filter(update_time__gt=update_time)
    if toutiaos.count() > 0:
        toutiao_update(toutiaos)


def fenghuang_create(fenghuangs):
    saveList = []
    r = fenghuangs.count() % 1000
    if fenghuangs.count() >= 1000:
        l = fenghuangs.count() // 1000
        for i in range(l):
            for fenghuang in fenghuangs[i * 1000:(i + 1) * 1000]:
                isexist = Advertisement.objects.filter(title=fenghuang.title, local_img=fenghuang.local_img,
                                                       update_time=fenghuang.update_time)  # 判断该数据是否已保存过
                if isexist.count() > 0:
                    continue
                advertisement = Advertisement()
                advertisement = fenghuang_object(fenghuang, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []
    if r != 0:
        for fenghuang in fenghuangs[len(fenghuangs) - r:len(fenghuangs)]:
            isexist = Advertisement.objects.filter(title=fenghuang.title, local_img=fenghuang.local_img,
                                                   update_time=fenghuang.update_time)  # 判断该数据是否已保存过
            if isexist.count() > 0:
                continue
            advertisement = Advertisement()
            advertisement = fenghuang_object(fenghuang, advertisement)
            if advertisement != None:
                saveList.append(advertisement)
        Advertisement.objects.bulk_create(saveList)


def fenghuang_update(fenghuangs):
    r = fenghuangs.count() % 1000
    if fenghuangs.count() >= 1000:
        l = fenghuangs.count() // 1000
        for i in range(l):
            for fenghuang in fenghuangs[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(url=fenghuang.weburl).filter(title=fenghuang.title)[0:1]
                if advertisement.count() > 1:
                    advertisement = fenghuang_object(fenghuang, advertisement[0])
                    if advertisement != None:
                        advertisement.save()

    if r != 0:
        for fenghuang in fenghuangs[len(fenghuangs) - r:len(fenghuangs)]:
            advertisement = Advertisement.objects.filter(url=fenghuang.weburl).filter(title=fenghuang.title)[0:1]
            if advertisement.count() > 1:
                advertisement = fenghuang_object(fenghuang, advertisement[0])
                if advertisement != None:
                    advertisement.save()


def sina_create(sinas):
    saveList = []
    r = sinas.count() % 1000
    if sinas.count() >= 1000:
        l = sinas.count() // 1000
        for i in range(l):
            for sina in sinas[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement()
                advertisement = sina_object(sina, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []
    if r != 0:
        for sina in sinas[sinas.count() - r:sinas.count()]:
            advertisement = Advertisement()
            advertisement = sina_object(sina, advertisement)
            if advertisement != None:
                saveList.append(advertisement)
        Advertisement.objects.bulk_create(saveList)


def sina_update(sinas):
    r = sinas.count() % 1000
    if sinas.count() >= 1000:
        l = sinas.count() // 1000
        for i in range(l):
            for sina in sinas[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(title=sina.title).filter(url=sina.link)
                if advertisement.count() < 1:
                    return None
                advertisement = sina_object(sina, advertisement[0])
                if advertisement != None:
                    advertisement.save()
    if r != 0:
        for sina in sinas[sinas.count() - r:sinas.count()]:
            advertisement = Advertisement.objects.filter(title=sina.title).filter(url=sina.link)
            if advertisement.count() < 1:
                return None
            advertisement = sina_object(sina, advertisement[0])
            if advertisement != None:
                advertisement.save()


def souhu_create(souhus):
    saveList = []
    r = souhus.count() % 1000
    if souhus.count() >= 1000:
        l = souhus.count() // 1000
        for i in range(l):
            for souhu in souhus[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement()
                advertisement = souhu_object(souhu, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []

        if r != 0:
            for souhu in souhus[souhus.count() - r:souhus.count()]:
                advertisement = Advertisement()
                advertisement = souhu_object(souhu, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)


def souhu_update(souhus):
    r = souhus.count() % 1000
    if souhus.count() >= 1000:
        l = souhus.count() // 1000
        for i in range(l):
            for souhu in souhus[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(title=souhu.resource_text).filter(url=souhu.link)
                if advertisement.count() > 0:
                    advertisement = souhu_object(souhu, advertisement[0])
                    if advertisement != None:
                        advertisement.save()

        if r != 0:
            for souhu in souhus[souhus.count() - r:souhus.count()]:
                advertisement = Advertisement.objects.filter(title=souhu.resource_text).filter(url=souhu.link)
                if advertisement.count() > 0:
                    advertisement = souhu_object(souhu, advertisement[0])
                    if advertisement != None:
                        advertisement.save()


@task
def get_feature(advertisement):
    advertisements = Advertisement.objects.exclude(local_img='暂无').exclude(local_img='')
    for advertisement in advertisements:
        try:
            feat = extract.extract_feat(
                '/repo/smspider/images/' + advertisement.model_from + '/' + advertisement.local_img)
            img_feat = feat.tolist()
            advertisement.image_feature = pickle.dumps(img_feat)
            advertisement.save()
        except Exception as e:
            print(e)


def feedsradar_create(feedsradarModel):
    saveList = []
    r = feedsradarModel.count() % 1000
    if feedsradarModel.count() >= 1000:
        l = feedsradarModel.count() // 1000
        for i in range(l):
            for feedsradar in feedsradarModel[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement()
                advertisement = feedsradar_object(feedsradar, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []
    if r != 0:
        for feedsradar in feedsradarModel[feedsradarModel.count() - r:feedsradarModel.count()]:
            advertisement = Advertisement()
            advertisement = feedsradar_object(feedsradar, advertisement)
            if advertisement != None:
                saveList.append(advertisement)
    Advertisement.objects.bulk_create(saveList)


def feedsradar_update(feedsradarModel):
    r = feedsradarModel.count() % 1000
    if feedsradarModel.count() >= 1000:
        l = feedsradarModel.count() // 1000
        for i in range(l):
            for feedsradar in feedsradarModel[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(title=feedsradar.title).filter(url=feedsradar.url)
                if advertisement.count() > 0:
                    advertisement = feedsradar_object(feedsradar, advertisement[0])
                    if advertisement != None:
                        advertisement.save()
    if r != 0:
        for feedsradar in feedsradarModel[feedsradarModel.count() - r:feedsradarModel.count()]:
            advertisement = Advertisement.objects.filter(title=feedsradar.title).filter(url=feedsradar.url)
            if advertisement.count() > 0:
                advertisement = feedsradar_object(feedsradar, advertisement[0])
                if advertisement != None:
                    advertisement.save()


def toutiao_create(toutiaos):
    saveList = []
    r = toutiaos.count() % 1000
    if toutiaos.count() >= 1000:
        l = toutiaos.count() // 1000
        for i in range(l):
            for toutiao in toutiaos[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement()
                advertisement = toutiao_object(toutiao, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []
        if r != 0:
            for toutiao in toutiaos[toutiaos.count() - r:toutiaos.count()]:
                advertisement = Advertisement()
                advertisement = toutiao_object(toutiao, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)


def toutiao_update(toutiaos):
    r = toutiaos.count() % 1000
    if toutiaos.count() >= 1000:
        l = toutiaos.count() // 1000
        for i in range(l):
            for toutiao in toutiaos[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(title=toutiao.title).filter(url=toutiao.article_url)
                if advertisement.count() > 0:
                    advertisement = toutiao_object(toutiao, advertisement[0])
                    if advertisement != None:
                        advertisement.save()

        if r != 0:
            for toutiao in toutiaos[toutiaos.count() - r:toutiaos.count()]:
                advertisement = Advertisement.objects.filter(title=toutiao.title).filter(url=toutiao.article_url)
                if advertisement.count() > 0:
                    advertisement = toutiao_object(toutiao, advertisement[0])
                    if advertisement != None:
                        advertisement.save()


def test_souhu(request):
    ad = SouhuModel.objects.filter(resource_text='INSPIRE 混动来袭')
    souhu = ad[0]
    advertisement = Advertisement()
    souhu_object(souhu, advertisement)
    return JsonResponse({'json_data': 'success'})


def radar_create(radars):
    saveList = []
    r = radars.count() % 1000
    if radars.count() >= 1000:
        l = radars.count() // 1000
        for i in range(l):
            for radar in radars[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement()
                advertisement = radar_object(radar, advertisement)
                if advertisement != None:
                    saveList.append(advertisement)
            Advertisement.objects.bulk_create(saveList)
            saveList = []
    if r != 0:
        for radar in radars[len(radars) - r:len(radars)]:
            advertisement = Advertisement()
            advertisement = radar_object(radar, advertisement)
            if advertisement != None:
                saveList.append(advertisement)
        Advertisement.objects.bulk_create(saveList)


def radar_update(radars):
    r = radars.count() % 1000
    if radars.count() >= 1000:
        l = radars.count() // 1000
        for i in range(l):
            for radar in radars[i * 1000:(i + 1) * 1000]:
                advertisement = Advertisement.objects.filter(title=radar.title).filter(url=radar.url)
                if advertisement.count() > 0:
                    advertisement = radar_object(radar, advertisement[0])
                    if advertisement != None:
                        advertisement.save()
    if r != 0:
        for radar in radars[len(radars) - r:len(radars)]:
            advertisement = Advertisement.objects.filter(title=radar.title).filter(url=radar.url)
            if advertisement.count() > 0:
                advertisement = radar_object(radar, advertisement[0])
                if advertisement != None:
                    advertisement.save()


def testdownload(request):
    image_path = 'E:/爬虫相关项目/pic/'
    souhuModels = SouhuModel.objects.all()
    for souhu in souhuModels:

        imgs = []
        filename = []
        img1 = json.loads(souhu.resource1.replace('\'', '"')).get('file')
        imgs.append(img1)

        img2 = ''
        img3 = ''
        try:
            if souhu.resource2 != '' or souhu.resource2 is not None:
                if str(souhu.resource2).find('file') >= 0:
                    img2 = json.loads(souhu.resource2.replace('\'', '"')).get('file')
                    imgs.append(img2)

            if souhu.resource3 != '' or souhu.resource3 is not None:
                if str(souhu.resource3).find('file') >= 0:
                    img3 = json.loads(souhu.resource3.replace('\'', '"')).get('file')
                    imgs.append(img3)
        except Exception as e:
            print(e)

        img_num = 1
        if len(imgs):
            try:
                for img in imgs:
                    if ".webp" in img:
                        img = img.replace(".webp", ".jpg")

                    if ".mp4" not in img:
                        ssl._create_default_https_context = ssl._create_unverified_context
                        local_img = image_path + '%s' % str(img.split('/')[-1])
                        urllib.request.urlretrieve(img, local_img)
                        img_num += 1
                        filename.append(img.split('/')[-1])
                souhu.local_img = str(filename)
                souhu.save()
            except Exception as e:
                print(e)

    return JsonResponse({'json_data': 'success'})
